"""
抓取免费代理网站代理
"""
import random
import time
from util.webRequest import WebRequest
from db.redisClient import RedisClient

from selenium import webdriver
from selenium.webdriver.ie.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By



def freeproxy01 (page_count=7):
    """
    云代理  "http://www.ip3366.net/"
    """
    start_url = "http://www.ip3366.net/free/"
    for page in range(1,page_count+1):
        page_url = start_url+f"?stype=1&page={page}"
        html_data = WebRequest().get(page_url).tree
        time.sleep(random.uniform(2,3))
        # print(html_data)
        trs = html_data.xpath("/html/body/div[2]/div/div[2]/table/tbody/tr")
        for tr in trs:
            ip = tr.xpath("./td[1]/text()")[0]
            port = tr.xpath("./td[2]/text()")[0]
            proxy = ip+":"+port
            # print(proxy)
            yield proxy

# 经理打电话了，不敢爬
# def freeproxy02 (page_count = 10):
#     """
#     快代理  "https://www.kuaidaili.com/"
#     """
#     start_url = "https://www.kuaidaili.com/free/"
#     # 配置无头模式(无界面浏览器)
#     option = webdriver.EdgeOptions()  # 创建一个配置对象
#     option.add_argument("--headless")  # 开启无界面模式
#     option.add_argument("--disable-gpu")  # 禁用gpu
#
#     web = webdriver.Edge(service=Service(executable_path=r'D:\Python\Python3.12.6\msedgedriver.exe'),options=option)
#     try:
#         for page in range(1,page_count+1):
#             page_url = start_url+f"inha/{page}/"
#             web.get(page_url)
#             # 等待页面内容加载出来
#             WebDriverWait(web,10).until(EC.presence_of_element_located((By.XPATH,'//*[@id="table__free-proxy"]/div/table/tbody/tr[12]/td[1]')))
#             time.sleep(random.uniform(2, 3))  # 随机随眠时间防反爬虫
#
#             tr_list = web.find_elements(By.XPATH,'//*[@id="table__free-proxy"]/div/table/tbody/tr')
#             for tr in tr_list:
#                 ip = tr.find_element(By.XPATH,'./td[1]').text
#                 port = tr.find_element(By.XPATH,'./td[2]').text
#                 proxy = ip+":"+port
#                 yield proxy
#     except Exception as e:
#         print("freeproxy02发生异常:",e)
#     finally:
#         web.close()


def freeproxy03(page_count = 10):
    """
    89免费代理  "https://www.89ip.cn/"
    """
    start_url = "https://www.89ip.cn/"
    # 配置无头模式(无界面浏览器)
    option = webdriver.EdgeOptions()  # 创建一个配置对象
    option.add_argument("--headless")  # 开启无界面模式
    option.add_argument("--disable-gpu")  # 禁用gpu

    web = webdriver.Edge(service=Service(executable_path=r'D:\Python\Python3.12.6\msedgedriver.exe'), options=option)
    try:
        for page in range(1, page_count + 1):
            page_url = start_url + f"index_{page}.html"
            web.get(page_url)
            # 等待页面内容加载出来
            WebDriverWait(web, 10)
            time.sleep(random.uniform(2, 3))  # 随机随眠时间防反爬虫

            mid = web.find_element(By.CLASS_NAME,'layui-table')
            tr_list = mid.find_elements(By.XPATH,'./tbody/tr')
            for tr in tr_list:
                ip = tr.find_element(By.XPATH, './td[1]').text
                port = tr.find_element(By.XPATH, './td[2]').text
                proxy = ip + ":" + port
                yield proxy
    except Exception as e:
        print("freeproxy03发生异常:", e)
    finally:
        web.close()

def freeproxy04 (page_count=10):
    """
    齐云代理ip  "https://proxy.ip3366.net/"
    """
    start_url = "https://proxy.ip3366.net/free/"
    for page in range(1,page_count+1):
        page_url = start_url+f"?action=china&page={page}"
        html_data = WebRequest().get(page_url).tree
        time.sleep(random.uniform(2,3))

        trs = html_data.xpath('//*[@id="content"]/section/div[2]/table/tbody/tr')
        for tr in trs:
            ip = tr.xpath("./td[1]/text()")[0]
            port = tr.xpath("./td[2]/text()")[0]
            proxy = ip+":"+port
            # print(proxy)
            yield proxy

proxy_func_list = [freeproxy04]

if __name__ == '__main__':
    client = RedisClient()
    for func in proxy_func_list:
        proxies = func()
        for one_proxy in proxies:
            # print("---代理写入数据库---")
            client.add(one_proxy)